package cn.oldsix.spark.utils;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/*
 * @Author : Wu.D.J
 * @Date : 2017/7/25
 * @Description :
 */
public class HDFSUploadFileUtil {

    private static FileSystem fileSystem = null;

    private static FileSystem localFileSystem = null;

    public static void uploadFile(String srcDirectory, String tagDirectory, String hdfsUrl, String regex) throws URISyntaxException, IOException {
        Configuration conf = new Configuration();
        URI uri = new URI(hdfsUrl);
        if (fileSystem == null) {
            fileSystem = FileSystem.get(uri, conf);
        }
        if (localFileSystem == null) {
            localFileSystem = FileSystem.getLocal(conf);
        }
        Path path = new Path(hdfsUrl + tagDirectory);
        if (fileSystem.exists(path)) {
            if (fileSystem.listStatus(path).length > 0) {
                fileSystem.close();
                throw new IllegalArgumentException("目标目录存在其他文件！");
            }
        } else {
            fileSystem.mkdirs(path);
        }
        FileStatus[] listStatus = null;
        if (regex.length() > 0) {
            listStatus = localFileSystem.globStatus(new Path(srcDirectory), new RegexAcceptPathFilter(regex));
        } else {
            listStatus = localFileSystem.globStatus(new Path(srcDirectory));
        }
        Path[] sources = FileUtil.stat2Paths(listStatus);
        for (Path p : sources) {
            fileSystem.copyFromLocalFile(p, path);
            System.out.println("文件[" + p.toString() + "]上传成功!");
        }
        fileSystem.close();
    }

    public static void main(String[] args) throws IOException, URISyntaxException {
        String source = "D:/workspace/myself/spark-example/src/main/java/cn/oldsix/spark/data/anagram/*";
        String target = "/data/spark_study/anagram/";
        String hdfs = "hdfs://oldsix1:9000";
        uploadFile(source, target, hdfs, "");
    }
}
